
# Put all the module import in this code chunk
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
import pandas as pd
from bs4 import BeautifulSoup
import requests
import numpy as np
import re
import seaborn as sns
from sklearn import tree
import matplotlib.pyplot as plt
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
from sklearn.metrics import accuracy_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.metrics import  r2_score, mean_squared_error
from sklearn.preprocessing import StandardScaler
import pylab as py
import plotly.express as px
import scipy.stats as stats
from scipy.stats import linregress
import statsmodels.api as sm
from sklearn.cluster import AgglomerativeClustering


def extract_soup(url):
    """ Extracts Pal names and href links from the given URL.

    Args:
      url (str): url of website in string format

    Returns:
      name_list (list): list of pal names
      href_list (list): list of pal links
    """
    # Get the HTML content from the provided URL
    html = requests.get(url).text

    # Parse the HTML with BeautifulSoup
    soup = BeautifulSoup(html, 'html.parser')

    # Find all Pal link elements
    pals_links = soup.find_all("a", class_="")

    # Initialize lists to store names and href links
    names_list = []
    href_list = []

    # Extract names and href links
    for link in pals_links:
        name_div = link.find("div", class_="name")
        if name_div:
            name = name_div.text.strip().lower()
            href = link['href']
            cleaned_name = re.sub(r'[^\w\s]', "", name)  # Remove non-alphanumeric characters
            cleaned_name = re.sub("\d+", "", cleaned_name)  # Remove digits
            cleaned_name = cleaned_name.replace(" ", "")  # Remove spaces
            names_list.append(cleaned_name)
            href_list.append(href)

    # Return the cleaned names and href links
    return names_list, href_list


def get_pal_details(pal_url):
    """ Fetches details of a Pal from the provided URL.

    Args:
      pal_url (str): url of individual pal

    Returns:
       stats_dict (dict): the name, level, rarity, and type stats of individual pal
    """
    try:
        response = requests.get(pal_url)
        if response.status_code == 404:
            raise ValueError("404 Not Found")

        pal_html = response.text
        pal_soup = BeautifulSoup(pal_html, 'html.parser')

        # Get the name of the Pal
        pal_name_tag = pal_soup.find("h1")
        if pal_name_tag:
            pal_name = pal_name_tag.text.strip()
        else:
            raise ValueError("Name not found")

        # Get the rarity and level of the Pal
        rarity_classes = ["epic rarity", "legendary rarity", "rare rarity", "common rarity"]
        pal_rarity = None
        pal_level = None
        for rarity_class in rarity_classes:
            rarity_div = pal_soup.find("div", class_=rarity_class)
            if rarity_div:
                level_tag = rarity_div.find("div", class_="lv")
                pal_level = int(re.sub(r'\D', '', level_tag.text.strip())) if level_tag else None
                rarity_tag = rarity_div.find("div", class_="name")
                pal_rarity = rarity_tag.text.strip() if rarity_tag else None
                break

        # Get the type (element) of the Pal
        elements_div = pal_soup.find("div", class_="elements")
        element_list = []
        if elements_div:
            element_items = elements_div.find_all("div", class_="element")
            element_list = [element.find("div", class_="name").text.strip() for element in element_items]

        # Get the stats of the Pal
        stats_div = pal_soup.find("div", class_="stats")
        stats_dict = {}
        if stats_div:
            stats_items = stats_div.find_all("div", class_="item")
            for stat in stats_items:
                stat_name_tag = stat.find("div", class_="name")
                stat_value_tag = stat.find("div", class_="value")
                if stat_name_tag and stat_value_tag:
                    stat_name = stat_name_tag.text.strip()
                    stat_value = int(stat_value_tag.text.strip())  # Convert to integer
                    stats_dict[stat_name] = stat_value

        return {
            'name': pal_name,
            'rarity': pal_rarity,
            'level': pal_level,
            **stats_dict,  # Unpack the stats_dict into the return dictionary
            'types': element_list
        }
    except Exception as e:
        print(f"Error processing {pal_url}: {e}")
        return {
            'name': "404 Not Found",
            'rarity': None,
            'level': None,
            **{stat: None for stat in stats_dict.keys()},
            'types': []
        }


def get_all_pals_details(href_list):
    """ Retrieves details for all Pals based on the list of href links.

    Args:
      href_list (list): list of pal href links

    Returns:
      all_pals_details (list): lift of all pal details
    """
    base_url = "https://palworld.gg"
    all_pals_details = []
    for href in href_list:
        pal_url = base_url + href
        pal_details = get_pal_details(pal_url)
        all_pals_details.append(pal_details)
    return all_pals_details


# URL of the Pals page
pals_stats_url = "https://palworld.gg/pals"

# Extract names and href links using the function
names_list, href_list = extract_soup(pals_stats_url)

# Get details for all Pals
all_pals_details = get_all_pals_details(href_list)

# Convert the collected data into a DataFrame
df = pd.DataFrame(all_pals_details)

# Display the DataFrame
df.head()


csv_path = 'all_pals_info.csv'
df.to_csv(csv_path, index=False)

df = pd.read_csv('all_pals_info.csv')


# Specify the columns to scale (excluding 'level')
columns_to_scale = ['HP', 'Defense', 'Crafting Speed', 'Melee Attack', 'Price',
                    'Shot Attack', 'Stamina', 'Support', 'Running Speed',
                    'Sprinting Speed', 'Slow Walk Speed']

# Initialize the StandardScaler
scaler = StandardScaler()

# Scale the specified columns
df[columns_to_scale] = scaler.fit_transform(df[columns_to_scale])

# Save the scaled dataset to a new CSV file
scaled_csv_path = 'scaled_all_pals_info.csv'
df.to_csv(scaled_csv_path, index=False)

df_pal = pd.read_csv('scaled_all_pals_info.csv')
df_pal.head()


def plot_feature_means_by_level(df, x_feat_list):
    """ Plots the mean values of specified features by level.

    Args:
      df (DataFrame): data frame containing the pals data.
      x_feat_list (list): List of features to plot.

    Returns:
      None: line plot of level vs mean value of pal features
    """
    # Define the range of levels to include in the plot
    levels_range = (1, 10)

    # Define the title of the plot
    title = 'Mean Values of Features by Level (Excluding Price)'

    # Calculate means for each feature by level
    means_by_level = df.groupby('level')[x_feat_list].mean()

    # Filter levels to be within the specified range
    filtered_levels = range(levels_range[0], levels_range[1] + 1)
    means_by_level_filtered = means_by_level.loc[filtered_levels]

    # Plot the features
    plt.figure(figsize=(10, 6))
    colors = plt.cm.get_cmap('tab10', len(x_feat_list))

    for i, feature in enumerate(x_feat_list):
        plt.plot(means_by_level_filtered.index, means_by_level_filtered[feature],
                 label=f'{feature}', linestyle='-', marker='o', color=colors(i))

    # Adding labels, title, and legend
    plt.xlabel('Level')
    plt.ylabel('Value')
    plt.title(title)
    plt.legend(loc='upper left')
    plt.show()


x_feat_list = ['HP', 'Defense', 'Melee Attack',
               'Shot Attack', 'Stamina', 'Support', 'Running Speed',
               'Sprinting Speed', 'Slow Walk Speed']

# Assuming df_pal is your DataFrame
plot_feature_means_by_level(df_pal, x_feat_list)

/var/folders/fb/ym0ddg2s2szgznymfq_5jfj40000gn/T/ipykernel_2122/178519516.py:26: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  colors = plt.cm.get_cmap('tab10', len(x_feat_list))


def plot_hp_distribution_by_rarity(csv_file):
    """ Creates an interactive box plot to visualize the distribution of Health
    Points (HP) across different rarity levels of Pals.

    Args
      csv_file (str): The path to the CSV file containing the Pals data. The CSV
        file must contain columns 'rarity', 'HP', 'name', and 'types'.

    Returns:
      None : Displays an interactive plot showing the distribution of HP by
        rarity, which also includes outliers and hover information for each data point.
    """

    # Example: plot_hp_distribution_by_rarity('all_pals_info.csv')

    # Load the data
    df_pals = pd.read_csv(csv_file)

    # Define the custom order for the rarity categories
    rarity_order = ['Common', 'Rare', 'Epic', 'Legendary']

    # Create the interactive box plot
    fig = px.box(df_pals, x='rarity', y='HP', points='outliers',
                 hover_data=['name', 'rarity', 'types'],
                 category_orders={'rarity': rarity_order},
                 title='HP Distribution by Rarity',
                 labels={'rarity': 'Rarity', 'HP': 'Health Points (HP)'})

    # Show the plot
    fig.update_layout(
        font=dict(
        size=18,
        color="black"))
    fig.show()

plot_hp_distribution_by_rarity('all_pals_info.csv')


def plot_price_vs_level_with_regression(csv_file, exclude_rarity='Legendary', label_size=18):
    """ Creates an interactive scatterplot of Price vs Level, excluding a specified rarity,
    and overlays a line of best fit derived from linear regression. The plot includes
    hover information and is color-coded by rarity.

    Args:
      csv_file (str): The path to the CSV file containing the Pals data. The CSV
        file must contain columns 'Price', 'level', 'rarity', 'name', and 'types'.
      exclude_rarity (str): (optional) The rarity level to exclude from the plot(default is 'Legendary').
      label_size (int): (optional) The font size for the plot labels (default is 18).

    Returns:
      None: Displays an interactive scatterplot with a line of best fit and prints
        the linear regression results including the slope, intercept, and R-squared value.
    """
    # Example: plot_price_vs_level_with_regression('all_pals_info.csv')


    # Load the data
    df_pals = pd.read_csv(csv_file)

    # Exclude specified rarity
    df_pals_filtered = df_pals[df_pals['rarity'] != exclude_rarity]

    # Perform linear regression to find the line of best fit
    slope, intercept, r_value, p_value, std_err = linregress(df_pals_filtered['Price'], df_pals_filtered['level'])

    # Calculate the line of best fit
    df_pals_filtered['Best Fit Line'] = intercept + slope * df_pals_filtered['Price']

    # Create the scatterplot with hover information, color-coded by rarity
    fig = px.scatter(df_pals_filtered, x='Price', y='level',
                     color='rarity',
                     hover_data={'name': True, 'rarity': True, 'types': True},
                     title=f'Scatterplot of Price vs Level (Excluding {exclude_rarity}) with Line of Best Fit',
                     labels={'Price': 'Price', 'level': 'Level (Rarity Number)'})

    # Add the line of best fit to the plot
    fig.add_traces(px.line(df_pals_filtered, x='Price', y='Best Fit Line').data)

    # Increase label size
    fig.update_layout(
        font=dict(size=label_size, color="black")
    )

    # Show the plot
    fig.show()

    # Print the linear regression results with more precision
    print(f"Slope: {slope:.5f}")
    print(f"Intercept: {intercept:.2f}")
    print(f"R-squared: {r_value**2:.4f}")


plot_price_vs_level_with_regression('all_pals_info.csv')

/var/folders/fb/ym0ddg2s2szgznymfq_5jfj40000gn/T/ipykernel_2122/3339407048.py:29: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

Slope: 0.00083
Intercept: 1.53
R-squared: 0.6866


def prepare_data(df, x_feat_list):
    """ Prepare the data by extracting features and adding a constant term.

    Args:
      df (DataFrame):
      x_feat_list (list): list of pal features

    Returns:
      X (DataFrame): input values
      y (series): output values
    """
    X = df[x_feat_list]
    X = sm.add_constant(X)
    y = df['level']
    return X, y

def fit_ols_model(X, y):
    """ Fit an OLS regression model and return the model summary and parameters.

    Args:
      X (DataFrame): input values
      y (series): ouput values

    Returns:
      model.summary() (statsmodels): OLS Regression Results (including coef, std error,
        and t values for the x features).
      model.params (float): coefficient values for the (x) features from the model
    """
    model = sm.OLS(y, X).fit()
    return model.summary(), model.params

def perform_kfold(X, y, n_splits=5, random_state = 42):
    """ Perform K-Fold cross-validation and return R-squared and MSE values.

    Args:
      X (DataFrame): input values
      y (series): output values

    Returns:
      r2_full (float): R-squared value
      mse_full (float): Mean squared error value
      y_pred_full (list): list of predicted rarity values
      valid_indices (tuple): the data only include for 1 to 10

    """
    # Filter the data to only include levels from 1 to 10
    valid_indices = np.where((y >= 1) & (y <= 10))
    X = X[valid_indices]
    y = y[valid_indices]

    # Initialize KFold
    kf = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)

    # Initialize array to store predictions
    y_pred_full = np.empty_like(y)

    # Loop through each split
    for train_index, test_index in kf.split(X):
        # Split the data into training and test sets
        X_train, X_test = X[train_index], X[test_index]
        y_train, y_test = y[train_index], y[test_index]

        # Fit the linear regression model
        reg = LinearRegression()
        reg.fit(X_train, y_train)

        # Predict on the test set
        y_pred = reg.predict(X_test)
        y_pred_full[test_index] = y_pred

    # Calculate R-squared and MSE values
    r2_full = r2_score(y, y_pred_full)
    mse_full = mean_squared_error(y, y_pred_full)

    return r2_full, mse_full, y_pred_full, valid_indices

def plot_residuals_vs_index(residuals):
    """ Plot the index vs. the errors (residuals).

    Args:
      residuals (np.arrays): errors

    Returns:
      None: scatterplot of index vs. the errors
    """
    plt.figure(figsize=(10, 6))
    plt.scatter(range(len(residuals)), residuals, alpha=0.5)
    plt.axhline(y=0, color='r', linestyle='--')
    plt.xlabel('Index')
    plt.ylabel('Residuals')
    plt.title('Index vs. Residuals')
    plt.show()

def plot_features_vs_residuals(df, x_feat_list, residuals, valid_indices):
    """ Plot each feature vs. the errors (residuals).

    Args:
      df (DataFrame): data frame of pals
      x_feat_list (list): list of str containing pal features
      residuals (np.array): the errors
      valid_indices (tuple): the data only include for 1 to 10

      Returns:
        None: plot of the pal fetaures vs errors
    """

    # filters the data to only include pals with levels 1- 10
    df_filtered = df.iloc[valid_indices]

    num_features = len(x_feat_list)
    num_rows = (num_features + 2) // 3  # Calculate rows needed for the given number of features (3 columns)

    fig, axes = plt.subplots(nrows=num_rows, ncols=3, figsize=(15, num_rows * 5))
    fig.suptitle('Feature vs. Residuals')

    for i, feature in enumerate(x_feat_list):
        row, col = divmod(i, 3)
        sns.scatterplot(x=df_filtered[feature], y=residuals, ax=axes[row, col])
        axes[row, col].axhline(y=0, color='r', linestyle='--')
        axes[row, col].set_xlabel(feature)
        axes[row, col].set_ylabel('Residuals')

    plt.tight_layout(rect=[0, 0.06, 1, 0.95])
    plt.show()

def plot_qq(residuals):
    """ Plot the normal probability quantile-quantile plot of the errors (residuals).

    Args:
      residuals (np.array): the errors

    Returns:
      None: a normal probability quantile-quantile plot
    """
    plt.figure(figsize=(10, 6))
    stats.probplot(residuals, dist="norm", plot=plt)
    plt.title('Normal Probability Q-Q Plot of Residuals')
    plt.show()


# 1. Prepare the data
x_feat_list = ['HP', 'Defense', 'Melee Attack',
               'Shot Attack', 'Stamina', 'Support', 'Running Speed',
               'Sprinting Speed', 'Slow Walk Speed']

X, y = prepare_data(df_pal, x_feat_list)

# 2. Fit the OLS model and display results
model_summary, model_params = fit_ols_model(X, y)
#print(model_summary)
print(model_params)

# 3. Perform K-Fold cross-validation and get the results
r2_full, mse_full, y_pred_full, valid_indices = perform_kfold(X.values, y.values)

print(f"R-squared value: {r2_full:.3f}")
print(f"mean square error: {mse_full:.3f}")

const              5.707317
HP                 1.525470
Defense            0.802142
Melee Attack      -0.506768
Shot Attack        1.674647
Stamina           -0.161568
Support           -0.123691
Running Speed      0.480275
Sprinting Speed    0.296393
Slow Walk Speed    0.172998
dtype: float64
R-squared value: 0.782
mean square error: 1.701


# Calculate residuals
residuals = y.values[valid_indices] - y_pred_full

# 4. Plot index vs. residuals
plot_residuals_vs_index(residuals)

# 5. Plot each feature vs. residuals
plot_features_vs_residuals(df_pal, x_feat_list, residuals, valid_indices)

# 6. Plot the Q-Q plot of residuals
plot_qq(residuals)


def convert_to_list(y_val):
    """ Converts the list of (predicted) rarity values to proper list format
    Args:
        y_val (np.array): predicted rarity values

    Returns:
        y_list (list): list of predicted
    """

    # converts the array to list format
    y_arr = np.array(y_val)
    y_list_full = str(y_arr.tolist())
    y_list = re.findall('[A-Z][^A-Z]*', y_list_full)

    # cleans the elements
    for y in range(len(y_list) - 1):
        # removes the extra 4 characters from each element
        y_list[y] = y_list[y][:-4]

    # removes the extra two characters from the last element
    y_list[-1] = y_list[-1][:-2]
    return y_list

def convert_to_binary(rarity_list, label):
    """Converts the list of rarity types to binary based on label
    Args:
        rarity_list (list): input integer
        label (str): rarity label

    Returns:
        bin_list (list): list of binary values
    """
    return [1 if val == label else 0 for val in rarity_list]

def get_acc_sens_spec(y_true, y_pred):
    """ computes sensitivity & specificity (assumed binary inputs)

    Args:
        y_true (np.array): binary ground truth per trial
        y_pred (np.array): binary prediction per trial

    Returns:
        acc (float): accuracy
        sens (float): sensitivity
        spec (float): specificity
    """
    tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=(0, 1)).ravel()

    # Compute sensitivity
    sens = tp / (tp + fn) if tp + fn else np.nan

    # Compute specificity
    spec = tn / (tn + fp) if tn + fp else np.nan

    # Compute accuracy
    acc = (tp + tn) / (tn + fp + fn + tp)

    return acc, sens, spec

def get_acc_sens_spec_each_rarity(y_true, y_pred):
    """Computes accuracy, sensitivitym, & specificity for each rarity

    Args:
        y_true (np.array): true rarity
        y_pred (np.array): predicted rarity

    Returns:
        acc (float): accuracy
        sens (float): sensitivity
        spec (float): specificity
    """

    rarity_labels = ["Common", "Epic", "Legendary", "Rare"]

    for label in rarity_labels:
        # converts labels to binary
        y_true_bin = convert_to_binary(y_true, label)
        y_pred_bin = convert_to_binary(y_pred, label)

        # plots confusion matrix for each rarity label
        conf_matrix = confusion_matrix(y_true=y_true_bin, y_pred=y_pred_bin, labels=[0, 1])
        conf_mat_disp = ConfusionMatrixDisplay(conf_matrix, display_labels=["Not " + label, label])
        conf_mat_disp.plot()
        plt.title(f"Confusion Matrix for the {label} rarity")

        # gets acc, sens, & spec for each rarity label
        acc, sens, spec = get_acc_sens_spec(y_true_bin, y_pred_bin)
        print(f"For the {label} rarity:")
        print("Accuracy:", acc)
        print("Sensitivity:", sens)
        print("Specificity:", spec)

def train_and_evaluate_classifier(x, y, model, n_splits=10):
    """Trains and evaluates a classifier using Stratified K-Fold cross-validation.

    Args:
      x (np.array): values that will be used as the predicter (test & train data)
      y (np.array): true values that train the data
      model (str): the type of classifier
      n_splits (int): the number of how many times the model is cross-validated

    Returns:
      y_pred (np.array): the predicted y values
    """
    kfold = StratifiedKFold(n_splits=n_splits)
    y_pred = np.empty_like(y)

    for train_idx, test_idx in kfold.split(x, y):
        x_train, y_train = x[train_idx], y[train_idx]
        x_test = x[test_idx]

        model.fit(x_train, y_train)
        y_pred[test_idx] = model.predict(x_test)

    return y_pred

def plot_conf_matrix(y_true, y_pred, labels, title='Confusion Matrix', cmap='Blues'):
    """Plots the confusion matrix for the given true and predicted labels.

    Args:
      y_true (np.array): the true y values
      y_pred (np.array): the predicted y values
      labels (list): list of matrix labels (the unique values found in y_true)
      title (str): title of matrix
      cmap (str): sets color of matrix

    Returns:
      None: Displays the confusion matrix
    """
    conf_matrix = confusion_matrix(y_true, y_pred)
    con_disp = ConfusionMatrixDisplay(confusion_matrix=conf_matrix, display_labels=labels)
    con_disp.plot(cmap=cmap)
    plt.title(title)
    plt.show()

def plot_decision_tree(model, feature_names, class_names, figsize=(15, 10)):
    """ Plots the decision tree for the provided model.

    Args:
      model (model): The decision tree model to plot.
      feature_names (list): List of feature names.
      class_names (list): List of class names.
      figsize (tuple): Size of the figure (default is (15, 10)).

    Returns:
      None: Displays the decision tree plot.
    """
    plt.figure(figsize=figsize)
    tree.plot_tree(model, feature_names=feature_names, class_names=class_names)
    plt.show()


# Define features and labels
x_feat_list = ["HP", "Defense", "Crafting Speed", "Melee Attack",
               "Shot Attack", "Stamina", "Support", "Running Speed",
               "Sprinting Speed", "Slow Walk Speed"]
x = df_pal.loc[:, x_feat_list].values
y = df_pal.loc[:, "rarity"].values

# Initialize the Decision Tree classifier
dec_tree_clf = tree.DecisionTreeClassifier(max_depth=3, random_state=42)

# Train and evaluate the classifier
y_pred_tree = train_and_evaluate_classifier(x, y, dec_tree_clf)

# Plot the decision tree
plot_decision_tree(dec_tree_clf, feature_names=x_feat_list, class_names=dec_tree_clf.classes_.tolist())

# Plot confusion matrix and calculate accuracy
plot_conf_matrix(y, y_pred_tree, labels=dec_tree_clf.classes_.tolist(), title='Confusion Matrix for Decision Tree Classifier')
print("Overall Accuracy:", accuracy_score(y, y_pred_tree))

# Get accuracy, sensitivity, and specificity for each rarity
y_list = convert_to_list(y)
y_pred_list = convert_to_list(y_pred_tree)
get_acc_sens_spec_each_rarity(y_list, y_pred_list)

/Users/dannisjessy/anaconda3/lib/python3.11/site-packages/sklearn/model_selection/_split.py:725: UserWarning:

The least populated class in y has only 7 members, which is less than n_splits=10.

Overall Accuracy: 0.6890243902439024
For the Common rarity:
Accuracy: 0.8658536585365854
Sensitivity: 0.8153846153846154
Specificity: 0.898989898989899
For the Epic rarity:
Accuracy: 0.7865853658536586
Sensitivity: 0.7027027027027027
Specificity: 0.8110236220472441
For the Legendary rarity:
Accuracy: 0.9878048780487805
Sensitivity: 0.7142857142857143
Specificity: 1.0
For the Rare rarity:
Accuracy: 0.7378048780487805
Sensitivity: 0.5272727272727272
Specificity: 0.8440366972477065


# Write the code to run functions to fit each model in separate code chunks.
# Interpret the model results.
def plot_feat_import(feat_list, feat_import, sort=True, limit=None):
    """Plots feature importances in a horizontal bar chart.

    Args:
      feat_list (list): str names of features
      feat_import (np.array): feature importances (mean gini reduce)
      sort (bool): if True, sorts features in decreasing importance
          from top to bottom of plot
      limit (int): if passed, limits the number of features shown
          to this value

    Returns:
      None: barchart of important features
    """
    if sort:
        idx = np.argsort(feat_import).astype(int)
        feat_list = [feat_list[_idx] for _idx in idx]
        feat_import = feat_import[idx]

    if limit is not None:
        feat_list = feat_list[-limit:]
        feat_import = feat_import[-limit:]

    plt.barh(feat_list, feat_import)
    plt.gcf().set_size_inches(5, len(feat_list) / 2)
    plt.xlabel('Feature importance\n(Mean decrease in Gini across all Decision Trees)')
    plt.show()


# Initialize the Random Forest classifier
max_depth = 3
rf_clf = RandomForestClassifier(max_depth = max_depth, n_estimators = 500, random_state = 42)


# Train and evaluate the classifier
#y_pred_rf = train_and_evaluate_classifier(x, y, rf_clf)
rf_clf.fit(x,y)
y_pred_rf = rf_clf.predict(x)
y_pred_rf

# Plot confusion matrix and calculate accuracy
plot_conf_matrix(y, y_pred_rf, labels=rf_clf.classes_, title='Confusion Matrix for Random Forest Classifier')
print("Overall Accuracy:", accuracy_score(y, y_pred_rf))

# Get accuracy, sensitivity, and specificity for each rarity
y_pred_rf_list = convert_to_list(y_pred_rf)
get_acc_sens_spec_each_rarity(y_list, y_pred_rf_list)

Overall Accuracy: 0.8597560975609756
For the Common rarity:
Accuracy: 0.9390243902439024
Sensitivity: 0.8769230769230769
Specificity: 0.9797979797979798
For the Epic rarity:
Accuracy: 0.9207317073170732
Sensitivity: 0.8378378378378378
Specificity: 0.9448818897637795
For the Legendary rarity:
Accuracy: 0.9817073170731707
Sensitivity: 0.5714285714285714
Specificity: 1.0
For the Rare rarity:
Accuracy: 0.8780487804878049
Sensitivity: 0.8909090909090909
Specificity: 0.8715596330275229


# Plot feature importances
plot_feat_import(x_feat_list, rf_clf.feature_importances_, limit=8)


# Define features and labels
x_feat_list_5 = ["HP", "Defense", "Shot Attack", "Slow Walk Speed", "Sprinting Speed"]
x_5 = df_pal.loc[:, x_feat_list_5].values

# Initialize the Decision Tree classifier
dec_tree_clf_5 = tree.DecisionTreeClassifier(max_depth=3, random_state=42)

# Train and evaluate the classifier
y_pred_tree_5_feat = train_and_evaluate_classifier(x_5, y, dec_tree_clf_5)

# Plot the decision tree
plt.figure(figsize=(15, 10))
tree.plot_tree(dec_tree_clf_5, feature_names=x_feat_list_5, class_names=dec_tree_clf_5.classes_.tolist())
plt.show()

# Plot confusion matrix and calculate accuracy
plot_conf_matrix(y, y_pred_tree_5_feat, labels=dec_tree_clf_5.classes_, title='Confusion Matrix for Decision Tree Classifier (Top 5 Features)')
print("Overall Accuracy:", accuracy_score(y, y_pred_tree_5_feat))

# Get accuracy, sensitivity, and specificity for each rarity
y_pred_tree_5_feat_list = convert_to_list(y_pred_tree_5_feat)
get_acc_sens_spec_each_rarity(y_list, y_pred_tree_5_feat_list)

/Users/dannisjessy/anaconda3/lib/python3.11/site-packages/sklearn/model_selection/_split.py:725: UserWarning:

The least populated class in y has only 7 members, which is less than n_splits=10.

Overall Accuracy: 0.6951219512195121
For the Common rarity:
Accuracy: 0.8719512195121951
Sensitivity: 0.8153846153846154
Specificity: 0.9090909090909091
For the Epic rarity:
Accuracy: 0.7926829268292683
Sensitivity: 0.7297297297297297
Specificity: 0.8110236220472441
For the Legendary rarity:
Accuracy: 0.9939024390243902
Sensitivity: 0.8571428571428571
Specificity: 1.0
For the Rare rarity:
Accuracy: 0.7317073170731707
Sensitivity: 0.509090909090909
Specificity: 0.8440366972477065


# Initialize the Decision Tree classifier for max_depth = 4
dec_tree_clf_5 = tree.DecisionTreeClassifier(max_depth=4, random_state=42)

# Train and evaluate the classifier
y_pred_tree_5_feat = train_and_evaluate_classifier(x_5, y, dec_tree_clf_5)

# Plot the decision tree
plt.figure(figsize=(15, 10))
tree.plot_tree(dec_tree_clf_5, feature_names=x_feat_list_5, class_names=dec_tree_clf_5.classes_.tolist())
plt.show()

# Plot confusion matrix and calculate accuracy
plot_conf_matrix(y, y_pred_tree_5_feat, labels=dec_tree_clf_5.classes_, title='Confusion Matrix for Decision Tree Classifier (Top 5 Features)')
print("Overall Accuracy:", accuracy_score(y, y_pred_tree_5_feat))

# Get accuracy, sensitivity, and specificity for each rarity
y_pred_tree_5_feat_list = convert_to_list(y_pred_tree_5_feat)
get_acc_sens_spec_each_rarity(y_list, y_pred_tree_5_feat_list)

/Users/dannisjessy/anaconda3/lib/python3.11/site-packages/sklearn/model_selection/_split.py:725: UserWarning:

The least populated class in y has only 7 members, which is less than n_splits=10.

Overall Accuracy: 0.725609756097561
For the Common rarity:
Accuracy: 0.8841463414634146
Sensitivity: 0.8
Specificity: 0.9393939393939394
For the Epic rarity:
Accuracy: 0.8292682926829268
Sensitivity: 0.5945945945945946
Specificity: 0.8976377952755905
For the Legendary rarity:
Accuracy: 0.9939024390243902
Sensitivity: 0.8571428571428571
Specificity: 1.0
For the Rare rarity:
Accuracy: 0.7439024390243902
Sensitivity: 0.7090909090909091
Specificity: 0.7614678899082569

	name	rarity	level	HP	Defense	Crafting Speed	Melee Attack	Shot Attack	Price	Stamina	Support	Running Speed	Sprinting Speed	Slow Walk Speed	types
0	Anubis	Epic	10	120	100	100	130	130	4960	100	100	800	1000	80	[Earth]
1	Arsox	Common	4	85	95	100	100	95	3520	100	100	600	800	87	[Fire]
2	Astegon	Epic	9	100	125	100	100	125	8200	300	100	600	800	100	[Dragon, Dark]
3	Azurobe	Rare	7	110	100	100	70	100	5600	100	100	600	800	75	[Water, Dragon]
4	Beakon	Rare	6	105	80	100	100	115	7490	160	100	750	1200	100	[Electricity]

	name	rarity	level	HP	Defense	Melee Attack	Shot Attack	Price	Stamina	Support	Running Speed	Sprinting Speed	Slow Walk Speed	types
0	Anubis	Epic	10	1.280843	0.525005	1.853478	1.731345	0.179085	-0.291512	0.026534	1.010247	0.739855	0.054645	['Earth']
1	Arsox	Common	4	-0.465484	0.262503	0.242327	0.041222	-0.319997	-0.291512	0.026534	0.152021	0.139879	0.193435	['Fire']
2	Astegon	Epic	9	0.282942	1.837518	0.242327	1.489899	1.302020	4.054663	0.026534	0.152021	0.139879	0.451187	['Dragon', 'Dark']
3	Azurobe	Rare	7	0.781892	0.525005	-1.368823	0.282669	0.400899	-0.291512	0.026534	0.152021	0.139879	-0.044490	['Water', 'Dragon']
4	Beakon	Rare	6	0.532417	-0.525005	0.242327	1.007007	1.055945	1.012341	0.026534	0.795690	1.339831	0.451187	['Electricity']

Pals: rarity¶

Team member: Enxi, Allie, Brey (Group 2)¶

Introduction¶

Data¶

Data overview¶

Import libraries¶

Webscraping and cleaning¶

Saving data into csv files¶

Visualizations¶

visualization 1¶

Interpretation of the Figure¶

visualization 2¶

Interpretation of the Box Plot¶

visualization 3¶

Interpretation of the Scatterplot¶

Models¶

Model 1: Multiple regression¶

Interpretation of Model 1(Multiple regression)¶

Model 2: Decision Tree model¶

Interpretation for Model 2 (Decision tree model):¶

Model 3: Random Forest and decision tree for top importances¶

Interpretation for Model 3¶

Random Forest Model:¶

Decion Tree With Top Five Features & Max Depth = 3:¶

Decision Tree With Top Five Features & Max Depth = 4:¶

Discussion¶

Labor Division and ideas on other group¶